package cn.wlh.learn.webmagic.selenium;

import cn.wlh.learn.webmagic.pipeline.DBPipeline;
import org.openqa.selenium.By;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.chrome.ChromeDriver;
import us.codecraft.webmagic.*;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;

import java.io.IOException;
import java.util.List;
import java.util.function.Consumer;

public class TouBiaoSelenium implements PageProcessor {
    private Site site = Site
            .me()
            .setRetryTimes(3)
            .setSleepTime(10000)
            .setRetrySleepTime(10000)
            .setTimeOut(10000)
            .addHeader("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3")
            .addHeader("Connection", "keep-alive")
            .addHeader("X-Requested-With", "XMLHttpRequest")
            .addHeader("Content-Type",
                    "application/x-www-form-urlencoded;charset=utf-8")
            .addHeader(
                    "User-Agent",
                    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36");

    String xPath ;
    String newstimeXpath ;
    String titleXpath ;

    public TouBiaoSelenium(String xPath, String newstimeXpath, String titleXpath) {
        this.xPath = xPath;
        this.newstimeXpath = newstimeXpath;
        this.titleXpath = titleXpath;
    }

    @Override
    public void process(Page page) {
        Html html = page.getHtml();
        //
        Selectable xpath = html.xpath(xPath);
        Selectable links = xpath.links();
        List<String> all = links.all();
        // 放入子集递归查询。
        page.addTargetRequests(all);

        page.getResultItems();
        //
        page.putField("keyword",KEY_WORD);
        page.putField("newstime",KEY_WORD);
        page.putField("title",KEY_WORD);
    }

    @Override
    public Site getSite() {
        return site;
    }
    // 查询关键字.
    private static final String KEY_WORD = "大屏";

    // 插入到。sheji_caiji_piaochong 表中.
    private static final DBPipeline dBPipeline = new DBPipeline("sheji_caiji_zhaobiao");
    public static void main(String[] args) throws Exception {



        //  湖南公共资源交易中心 https://www.hnsggzy.com/jygk/index.jhtml
        new TouBiaoSelenium("/html/body/div[4]/div[3]/div/ul",
                              "/html/body/div[4]/div[3]/div/ul/li/div[@class='article-list3-t']/div/text()",
                              "/html/body/div[4]/div[3]/div/ul/li/div/a/text()"  )
                .execHttp("https://www.hnsggzy.com/queryContent-jygk.jspx?title=" + KEY_WORD + "&origin=&inDates=&channelId=845&ext=&beginTime=&endTime=");


        // 北京公共资源交易中心 https://ggzyfw.beijing.gov.cn/jylcgcjs/index.html
        new TouBiaoSelenium("//p[@class='cs_search_title']",
                "//p[@class='cs_search_content_time']/span[2]/text()"
                ,"//p[@class='cs_search_title']/span[2]/a/span/text()")
                .execHttp("https://ggzyfw.beijing.gov.cn/elasticsearch/index.jsp?qt="+KEY_WORD);


        //  中国政府采购网  http://www.ccgp.gov.cn/xxgg/  TODO 太大了，近3天的。
        new TouBiaoSelenium("/html/body/div[5]/div[2]/div/div/div[1]/ul/li/a",
                "/html/body/div[5]/div[2]/div/div/div[1]/ul/li/span/text()",
                "/html/body/div[5]/div[2]/div/div/div[1]/ul/li/a/text()")
                .execHttp("http://search.ccgp.gov.cn/bxsearch?searchtype=2&page_index=1&bidSort=&buyerName=&projectId=&pinMu=&bidType=&dbselect=bidx&kw="+KEY_WORD+"&timeType=1&displayZone=&zoneId=&pppStatus=0&agentName=");


        // 公共资源交易中心（公共全国性的）http://deal.ggzy.gov.cn/ds/deal/dealList.jsp
            // 数据来源:省平台
            new TouBiaoSelenium("//*[@id='toview']/div/div/h4/a",
                    "//*[@id='toview']/div/div/h4/span/text()",
                    "//*[@id='toview']/div/div/h4/a/text()")
                    .execHttp("http://deal.ggzy.gov.cn/ds/deal/dealList.jsp",(driver) -> {

                        // 模拟点击 业务类型-不限
                        driver.findElement(By.xpath("//*[@id='choose_classify_00']/a"))
                                .click();
                        driver.findElement(By.id("FINDTXT")).sendKeys(KEY_WORD); // 用户名
                        // 点击搜索
                        driver.findElement(By.id("searchButton")).click();
                    });
            //  数据来源:央企招投标
            new TouBiaoSelenium("//*[@id='toview']/div/div/h4/a",
                    "//*[@id='toview']/div/div/h4/span/text()",
                    "//*[@id='toview']/div/div/h4/a/text()")
                    .execHttp("http://deal.ggzy.gov.cn/ds/deal/dealList.jsp",(driver) -> {

                        // 模拟点击 业务类型-不限
                        driver.findElement(By.xpath("//*[@id='choose_source_2']/a"))
                                .click();
                        driver.findElement(By.id("FINDTXT")).sendKeys(KEY_WORD); // 用户名
                        // 点击搜索
                        driver.findElement(By.id("searchButton")).click();
                    });

        // 必联（电子招标多） https://www.ebnew.com/
        new TouBiaoSelenium("/html/body/div[2]/div[4]/div[1]/div[2]/div/div[1]/a",
                "/html/body/div[2]/div[4]/div[1]/div[2]/div/div[1]/i[2]/text()",
                "/html/body/div[2]/div[4]/div[1]/div[2]/div/div[1]/a/text()")
                .execHttp("https://ss.ebnew.com/tradingSearch/index.htm?searhType=1&key="+KEY_WORD);








//        execSelenium(new HnsggzySelenium("/html/body/div[4]/div[3]/div/ul"),
//                url);

    }

    private static void execSelenium(TouBiaoSelenium dome, String url) {
        Spider.create(dome).addUrl(url)
                .addPipeline(dBPipeline)
                .thread(1).start();
    }
    // 默认没有操作
    private  void execHttp(String url) throws IOException {
        execHttp(url,(x) -> {});
    }
    private  void execHttp(String url, Consumer<WebDriver> caozuo) throws IOException {
        // 请求网络.
//        Document parse = Jsoup.parse(new URL(url), 10 * 1000);
        System.setProperty("webdriver.chrome.driver",
                "D:/chromedriver.exe"); // 注册驱动
        WebDriver driver = new ChromeDriver();


        driver.get(url);// 打开网址

        // 页面模拟操作。
        caozuo.accept(driver);

        String parse = driver.getPageSource();
        driver.close();

//        String parse =  HttpRequest.get(url).timeout(-1)
//                .header("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3")
//                .header("Connection", "keep-alive")
//                .header("X-Requested-With", "XMLHttpRequest")
//                .header("Content-Type",
//                        "application/x-www-form-urlencoded;charset=utf-8")
//                .header(
//                        "User-Agent",
//                        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36")
//                .execute().body();;

        Html html = new Html(parse);
        // 获得连接地址.
        Selectable xpath = html.xpath(xPath);
        Selectable links = xpath.links();
        List<String> linkAll = links.all();
        // 获得时间 /html/body/div[4]/div[3]/div/ul/li[1]/div[1]/a

        Selectable newstimeSelectable = html.xpath(newstimeXpath );
        List<String> newstimeLinkAll = newstimeSelectable.all();
        // 获得标题

        // /html/body/div[4]/div[3]/div/ul/li[1]/div[1]/div

        Selectable titleSelectable = html.xpath(titleXpath);
        List<String> titleLinkAll = titleSelectable.all();
        for (int i = 0; i < linkAll.size(); i++) {
            String link  = linkAll.get(i);
            String newstime = newstimeLinkAll.get(i);
            String title = titleLinkAll.get(i);

            // 放入url -> id。
            ResultItems resultItems = new ResultItems();
            Request request = new Request();
            request.setUrl(link);
            resultItems.setRequest(request);
            // 放入表中其他属性.
            resultItems.put("keyword",KEY_WORD);
            resultItems.put("newstime",newstime);
            resultItems.put("title",title);
            dBPipeline.process(resultItems,null);
        }
    }
}
